Packages

library(plotly)
library(htmlwidgets) # for saving figures (with saveWidget function)
library(htmltools)

A/B Test

A/B Test - Definition

# Define minimum set of variables that fully define the A/B test results
t = 20     # Total number of subjects (A + B)
a = 10     # Number of subjects in group A
b = 10     # Number of subjects in group B
a_yes = 7  # Number of successes (yes) in group A
b_yes = 4  # Number of successes (yes) in group B

# Compute remaining A/B test results (using the variables above)
a_no = a - a_yes                # Number of failures (no) in group A
b_no = b - b_yes                # Number of failures (no) in group B
t_yes = a_yes + b_yes           # Total (A + B) successes (yes)
t_no  = a_no  + b_no            # Total (A + B) failures (no)
a_yes_pc = 100 * a_yes / a      # Percentage of successes (yes) in group A
b_yes_pc = 100 * b_yes / b      # Percentage of successes (yes) in group B
ab_yes_pc = a_yes_pc - b_yes_pc # Test statistic: Yes percentage diff. (A - B)

cat('Yes Rate (%):\n   A:', a_yes_pc, '\n   B:', b_yes_pc, '\n A-B:', ab_yes_pc)
Yes Rate (%):
   A: 70 
   B: 40 
 A-B: 30

A/B Test - Dataframe

set.seed(0) # For reproducible results

sub_rand = sample(1:t) # Subject randomization (for step 2)

# A/B results (for step 3): -1 = No, 0 = NA, +1 = Yes
ab_res = c(sample(rep(c(+1, -1), c(a_yes, a_no))),
           sample(rep(c(+1, -1), c(b_yes, b_no))))

# Data frame for animations
ab_df = data.frame(
  # A/B steps:   Step 1,    Step 2,    Step 3
  step    = c(rep(1, t), rep(2, t), rep(3, t)), # A/B step
  sub_y   = c(      1:t,       1:t,       1:t), # Subject y position
  sub_id  = c(      1:t,  sub_rand,  sub_rand), # Subject ID
  sub_col = c(rep(0, t), rep(0, t),    ab_res)  # Subject color (A/B result)
)

ab_df

A/B Test - Create Base Fig

A/B Test - Create Steps Fig

A/B Test - Save Steps Fig

saveWidget(ab_step_fig, "ab-steps-fig.html")

A/B Test - iframe

A/B Test - htmltools::tags$iframe

  • Bad: Does not show in the R notebook.
  • Good: Shows in the presentation.

A/B Test - htmltools::includeHTML

  • Bad: Does not show in the presentation
  • Good: Shows in the R notebook

A/B Test - Create Subjects Fig

A/B Test - Save Subjects Fig

saveWidget(ab_sub_fig, "ab-subjects-fig.html")

Permutation Test

Permutation Test - Dataframe

set.seed(0) # For reproducible results

sub_id_1 = c(which(ab_res == 1), which(ab_res == -1))
sub_col_1 = c(rep(1, t_yes), rep(-1, t_no))

perm_rand = sample(1:t)
sub_id_2 = sub_id_1[perm_rand]
sub_col_2 = sub_col_1[perm_rand]

# Data frame for animations
perm_df = data.frame(
  # Perm steps:  Step 0,    Step 1,    Step 2
  step    = c(rep(0, t), rep(1, t), rep(2, t)), # Permutation step
  sub_y   = c(      1:t,       1:t,       1:t), # Subject y position
  sub_id  = c(      1:t,  sub_id_1,  sub_id_2), # Subject ID
  sub_col = c(   ab_res, sub_col_1, sub_col_2)  # Subject color (A/B result)
)

perm_df

Permutation Test - Create Base Fig

shapes = list(
  list(type="rect", x0=-0.2, x1=0.2, y0=0.5,  y1=10.5),
  list(type="rect", x0=-0.2, x1=0.2, y0=10.5,  y1=20.5),
  list(type="rect", x0=0.8, x1=1.2, y0=0.5, y1=20.5),
  list(type="rect", x0=1.8, x1=2.2, y0=0.5,  y1=10.5),
  list(type="rect", x0=1.8, x1=2.2, y0=10.5, y1=20.5)
)

annotations = list(
  list(x=-0.8, y=5.5, text="A: 70%", showarrow=FALSE, font=list(size=18)),
  list(x=-0.8, y=15.5, text="B: 40%", showarrow=FALSE, font=list(size=18)),
  list(x=-0.8, y=10.5, text="A-B: 30%", showarrow=FALSE, font=list(size=18)),
  list(x=2.8, y=5.5, text="A: 50%", showarrow=FALSE, font=list(size=18)),
  list(x=2.8, y=15.5, text="B: 60%", showarrow=FALSE, font=list(size=18)),
  list(x=2.8, y=10.5, text="A-B: -10%", showarrow=FALSE, font=list(size=18))
)

x_labels = list("Step 0\nA/B Results", 'Step 1\n"Bag"', "Steps 2-5\nOne Permutation")
x_axis = list(title="", range = c(-1.3, 3.4), tickvals = list(0, 1, 2),
              ticktext = x_labels, zeroline=FALSE, showgrid=FALSE)
y_axis = list(title="", range = c(0, 21), zeroline=FALSE, showline=FALSE,
              showticklabels=FALSE, showgrid = FALSE)

perm_base_fig = perm_df %>%
  plot_ly(x=~step, y=~sub_y, color=~sub_col, colors=c("red", "black", "green3")) %>%
  add_trace(type="scatter", mode="markers", marker=list(size=10), opacity=0.3) %>%
  config(displayModeBar=FALSE) %>%
  layout(annotations=annotations, margin=list(l=0, r=0, b=0, t=0, pad=0), 
         shapes=shapes, xaxis=x_axis, yaxis=y_axis)

perm_base_fig

Permutation Test - Create Steps Fig

perm_step_fig = perm_base_fig %>%
  add_trace(type="scatter", mode="markers", marker=list(size=10), frame=~step, ids=~sub_id) %>%
  hide_colorbar() %>%
  layout(showlegend=FALSE)

perm_step_fig

Permutatio Test - Save Steps Fig

saveWidget(perm_step_fig, "permutation-steps-fig.html")

Permutation Test - Create Subjects Fig

perm_sub_fig = perm_base_fig %>%
  add_trace(type="scatter", mode="markers+lines", marker=list(size=10), frame=~sub_id) %>%
  hide_colorbar() %>%
  layout(showlegend=FALSE)

perm_sub_fig

Permutatio Test - Save Subjects Fig

saveWidget(perm_sub_fig, "permutation-subjects-fig.html")
LS0tDQp0aXRsZTogIkRpYWdyYW1zIg0Kb3V0cHV0OiBodG1sX25vdGVib29rDQotLS0NCg0KYGBge3IsIGluY2x1ZGU9RkFMU0V9DQoja25pdHI6Om9wdHNfY2h1bmskc2V0KGVjaG8gPSBGQUxTRSkNCmBgYA0KDQojIyBQYWNrYWdlcw0KDQpgYGB7ciwgbWVzc2FnZT1GQUxTRX0NCmxpYnJhcnkocGxvdGx5KQ0KbGlicmFyeShodG1sd2lkZ2V0cykgIyBmb3Igc2F2aW5nIGZpZ3VyZXMgKHdpdGggc2F2ZVdpZGdldCBmdW5jdGlvbikNCmxpYnJhcnkoaHRtbHRvb2xzKQ0KYGBgDQoNCiMgQS9CIFRlc3QNCg0KIyMgQS9CIFRlc3QgLSBEZWZpbml0aW9uIHsuc21hbGxlcn0NCg0KYGBge3J9DQojIERlZmluZSBtaW5pbXVtIHNldCBvZiB2YXJpYWJsZXMgdGhhdCBmdWxseSBkZWZpbmUgdGhlIEEvQiB0ZXN0IHJlc3VsdHMNCnQgPSAyMCAgICAgIyBUb3RhbCBudW1iZXIgb2Ygc3ViamVjdHMgKEEgKyBCKQ0KYSA9IDEwICAgICAjIE51bWJlciBvZiBzdWJqZWN0cyBpbiBncm91cCBBDQpiID0gMTAgICAgICMgTnVtYmVyIG9mIHN1YmplY3RzIGluIGdyb3VwIEINCmFfeWVzID0gNyAgIyBOdW1iZXIgb2Ygc3VjY2Vzc2VzICh5ZXMpIGluIGdyb3VwIEENCmJfeWVzID0gNCAgIyBOdW1iZXIgb2Ygc3VjY2Vzc2VzICh5ZXMpIGluIGdyb3VwIEINCg0KIyBDb21wdXRlIHJlbWFpbmluZyBBL0IgdGVzdCByZXN1bHRzICh1c2luZyB0aGUgdmFyaWFibGVzIGFib3ZlKQ0KYV9ubyA9IGEgLSBhX3llcyAgICAgICAgICAgICAgICAjIE51bWJlciBvZiBmYWlsdXJlcyAobm8pIGluIGdyb3VwIEENCmJfbm8gPSBiIC0gYl95ZXMgICAgICAgICAgICAgICAgIyBOdW1iZXIgb2YgZmFpbHVyZXMgKG5vKSBpbiBncm91cCBCDQp0X3llcyA9IGFfeWVzICsgYl95ZXMgICAgICAgICAgICMgVG90YWwgKEEgKyBCKSBzdWNjZXNzZXMgKHllcykNCnRfbm8gID0gYV9ubyAgKyBiX25vICAgICAgICAgICAgIyBUb3RhbCAoQSArIEIpIGZhaWx1cmVzIChubykNCmFfeWVzX3BjID0gMTAwICogYV95ZXMgLyBhICAgICAgIyBQZXJjZW50YWdlIG9mIHN1Y2Nlc3NlcyAoeWVzKSBpbiBncm91cCBBDQpiX3llc19wYyA9IDEwMCAqIGJfeWVzIC8gYiAgICAgICMgUGVyY2VudGFnZSBvZiBzdWNjZXNzZXMgKHllcykgaW4gZ3JvdXAgQg0KYWJfeWVzX3BjID0gYV95ZXNfcGMgLSBiX3llc19wYyAjIFRlc3Qgc3RhdGlzdGljOiBZZXMgcGVyY2VudGFnZSBkaWZmLiAoQSAtIEIpDQoNCmNhdCgnWWVzIFJhdGUgKCUpOlxuICAgQTonLCBhX3llc19wYywgJ1xuICAgQjonLCBiX3llc19wYywgJ1xuIEEtQjonLCBhYl95ZXNfcGMpDQpgYGANCg0KIyMgQS9CIFRlc3QgLSBEYXRhZnJhbWUgey5zbWFsbGVyfQ0KDQpgYGB7cn0NCnNldC5zZWVkKDApICMgRm9yIHJlcHJvZHVjaWJsZSByZXN1bHRzDQoNCnN1Yl9yYW5kID0gc2FtcGxlKDE6dCkgIyBTdWJqZWN0IHJhbmRvbWl6YXRpb24gKGZvciBzdGVwIDIpDQoNCiMgQS9CIHJlc3VsdHMgKGZvciBzdGVwIDMpOiAtMSA9IE5vLCAwID0gTkEsICsxID0gWWVzDQphYl9yZXMgPSBjKHNhbXBsZShyZXAoYygrMSwgLTEpLCBjKGFfeWVzLCBhX25vKSkpLA0KICAgICAgICAgICBzYW1wbGUocmVwKGMoKzEsIC0xKSwgYyhiX3llcywgYl9ubykpKSkNCg0KIyBEYXRhIGZyYW1lIGZvciBhbmltYXRpb25zDQphYl9kZiA9IGRhdGEuZnJhbWUoDQogICMgQS9CIHN0ZXBzOiAgIFN0ZXAgMSwgICAgU3RlcCAyLCAgICBTdGVwIDMNCiAgc3RlcCAgICA9IGMocmVwKDEsIHQpLCByZXAoMiwgdCksIHJlcCgzLCB0KSksICMgQS9CIHN0ZXANCiAgc3ViX3kgICA9IGMoICAgICAgMTp0LCAgICAgICAxOnQsICAgICAgIDE6dCksICMgU3ViamVjdCB5IHBvc2l0aW9uDQogIHN1Yl9pZCAgPSBjKCAgICAgIDE6dCwgIHN1Yl9yYW5kLCAgc3ViX3JhbmQpLCAjIFN1YmplY3QgSUQNCiAgc3ViX2NvbCA9IGMocmVwKDAsIHQpLCByZXAoMCwgdCksICAgIGFiX3JlcykgICMgU3ViamVjdCBjb2xvciAoQS9CIHJlc3VsdCkNCikNCg0KYWJfZGYNCmBgYA0KDQojIyBBL0IgVGVzdCAtIENyZWF0ZSBCYXNlIEZpZw0KDQpgYGB7ciwgZWNobz1GQUxTRSwgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCnNoYXBlcyA9IGxpc3QoDQogIGxpc3QodHlwZT0icmVjdCIsIHgwPTAuOCwgeDE9MS4yLCB5MD0wLjUsICB5MT0yMC41KSwNCiAgbGlzdCh0eXBlPSJyZWN0IiwgeDA9MS44LCB4MT0yLjIsIHkwPTAuNSwgIHkxPTEwLjUpLA0KICBsaXN0KHR5cGU9InJlY3QiLCB4MD0xLjgsIHgxPTIuMiwgeTA9MTAuNSwgeTE9MjAuNSksDQogIGxpc3QodHlwZT0icmVjdCIsIHgwPTIuOCwgeDE9My4yLCB5MD0wLjUsICB5MT0xMC41KSwNCiAgbGlzdCh0eXBlPSJyZWN0IiwgeDA9Mi44LCB4MT0zLjIsIHkwPTEwLjUsIHkxPTIwLjUpLA0KICBsaXN0KHR5cGU9InJlY3QiLCB4MD0yLjIsIHgxPTIuOCwgeTA9MC41LCAgeTE9MTAuNSwgZmlsbGNvbG9yPSJibHVlIiwgb3BhY2l0eT0wLjIpLA0KICBsaXN0KHR5cGU9InJlY3QiLCB4MD0yLjIsIHgxPTIuOCwgeTA9MTAuNSwgeTE9MjAuNSwgZmlsbGNvbG9yPSJ5ZWxsb3ciLCBvcGFjaXR5PTAuMikNCikNCg0KYW5ub3RhdGlvbnMgPSBsaXN0KA0KICBsaXN0KHg9Mi41LCB5PTUuNSwgdGV4dD0iQSIsIHNob3dhcnJvdz1GQUxTRSwgZm9udD1saXN0KHNpemU9MjApKSwNCiAgbGlzdCh4PTIuNSwgeT0xNS41LCB0ZXh0PSJCIiwgc2hvd2Fycm93PUZBTFNFLCBmb250PWxpc3Qoc2l6ZT0yMCkpLA0KICBsaXN0KHg9My42LCB5PTUuNSwgdGV4dD0iQTogNzAlIiwgc2hvd2Fycm93PUZBTFNFLCBmb250PWxpc3Qoc2l6ZT0xOCkpLA0KICBsaXN0KHg9My42LCB5PTE1LjUsIHRleHQ9IkI6IDQwJSIsIHNob3dhcnJvdz1GQUxTRSwgZm9udD1saXN0KHNpemU9MTgpKSwNCiAgbGlzdCh4PTMuNiwgeT0xMC41LCB0ZXh0PSJBLUI6IDMwJSIsIHNob3dhcnJvdz1GQUxTRSwgZm9udD1saXN0KHNpemU9MTgpKQ0KKQ0KDQp4X2xhYmVscyA9IGxpc3QoIlN0ZXAgMVxuQWxsIHN1YmplY3RzIiwgIlN0ZXAgMlxuUmFuZG9taXphdGlvbiIsICJTdGVwIDNcblJlc3VsdHMiKQ0KeF9heGlzID0gbGlzdCh0aXRsZT0iIiwgcmFuZ2UgPSBjKDAuNSwgNC4yKSwgdGlja3ZhbHMgPSBsaXN0KDEsIDIsIDMpLA0KICAgICAgICAgICAgICB0aWNrdGV4dCA9IHhfbGFiZWxzKQ0KeV9heGlzID0gbGlzdCh0aXRsZT0iIiwgcmFuZ2UgPSBjKDAsIDIxKSwgemVyb2xpbmU9RkFMU0UsIHNob3dsaW5lPUZBTFNFLA0KICAgICAgICAgICAgICBzaG93dGlja2xhYmVscz1GQUxTRSwgc2hvd2dyaWQgPSBGQUxTRSkNCg0KYWJfYmFzZV9maWcgPSBhYl9kZiAlPiUNCiAgcGxvdF9seSh4PX5zdGVwLCB5PX5zdWJfeSwgY29sb3I9fnN1Yl9jb2wsIGNvbG9ycz1jKCJyZWQiLCAiYmxhY2siLCAiZ3JlZW4zIikpICU+JQ0KICBhZGRfdHJhY2UobWFya2VyPWxpc3Qoc2l6ZT0xMCksIG9wYWNpdHk9MC4zKSAlPiUNCiAgY29uZmlnKGRpc3BsYXlNb2RlQmFyPUZBTFNFKSAlPiUNCiAgbGF5b3V0KGFubm90YXRpb25zPWFubm90YXRpb25zLCBtYXJnaW49bGlzdChsPTAsIHI9MCwgYj0wLCB0PTAsIHBhZD0wKSwgDQogICAgICAgICBzaGFwZXM9c2hhcGVzLCB4YXhpcz14X2F4aXMsIHlheGlzPXlfYXhpcykNCg0KYWJfYmFzZV9maWcNCmBgYA0KDQojIyBBL0IgVGVzdCAtIENyZWF0ZSBTdGVwcyBGaWcNCg0KYGBge3IsIGVjaG89RkFMU0UsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQojIChyIGNodW5rIG9wdGlvbjogZmlnLmhlaWdodD01LjUpDQphYl9zdGVwX2ZpZyA9IGFiX2Jhc2VfZmlnICU+JQ0KICBhZGRfdHJhY2UobWFya2VyPWxpc3Qoc2l6ZT0xMCksIGZyYW1lPX5zdGVwLCBpZHM9fnN1Yl9pZCkgJT4lDQogIGhpZGVfY29sb3JiYXIoKSAlPiUNCiAgbGF5b3V0KHNob3dsZWdlbmQ9RkFMU0UpDQoNCmFiX3N0ZXBfZmlnDQpgYGANCg0KIyMgQS9CIFRlc3QgLSBTYXZlIFN0ZXBzIEZpZw0KDQpgYGB7cn0NCnNhdmVXaWRnZXQoYWJfc3RlcF9maWcsICJhYi1zdGVwcy1maWcuaHRtbCIpDQpgYGANCg0KIyMgQS9CIFRlc3QgLSBgaWZyYW1lYA0KDQo8aWZyYW1lIHNyYz0iYWItc3RlcHMtZmlnLmh0bWwiPjwvaWZyYW1lPg0KDQo8IS0tDQo8aWZyYW1lIHNyYz0iYWItc3RlcHMtZmlnLmh0bWwiIHN0eWxlPSJib3JkZXI6IG5vbmU7Ij48L2lmcmFtZT4NCi0tPg0KDQojIyBBL0IgVGVzdCAtIGBodG1sdG9vbHM6OnRhZ3MkaWZyYW1lYA0KDQotIEJhZDogRG9lcyBub3Qgc2hvdyBpbiB0aGUgUiBub3RlYm9vay4NCi0gR29vZDogU2hvd3MgaW4gdGhlIHByZXNlbnRhdGlvbi4NCg0KYGBge3IsIGVjaG89RkFMU0V9DQojZmlsZV9wYXRoID0gImFiLXN0ZXBzLWZpZy5odG1sIg0KI2h0bWx0b29sczo6dGFncyRpZnJhbWUoc3JjID0gZmlsZV9wYXRoLCBzY3JvbGxpbmcgPSAibm8iLCANCiMgICAgICAgICAgICAgICAgICAgICAgIHNlYW1sZXNzID0gInNlYW1sZXNzIiwgZnJhbWVCb3JkZXIgPSAiMCIpDQpgYGANCg0KIyMgQS9CIFRlc3QgLSBgaHRtbHRvb2xzOjppbmNsdWRlSFRNTGANCg0KLSBCYWQ6IERvZXMgbm90IHNob3cgaW4gdGhlIHByZXNlbnRhdGlvbg0KLSBHb29kOiBTaG93cyBpbiB0aGUgUiBub3RlYm9vaw0KDQpgYGB7ciwgZWNobz1GQUxTRX0NCiNodG1sdG9vbHM6OmluY2x1ZGVIVE1MKCJhYi1zdGVwcy1maWcuaHRtbCIpDQpgYGANCg0KIyMgQS9CIFRlc3QgLSBDcmVhdGUgU3ViamVjdHMgRmlnDQoNCmBgYHtyLCBlY2hvPUZBTFNFLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KYWJfc3ViX2ZpZyA9IGFiX2Jhc2VfZmlnICU+JQ0KICBhZGRfdHJhY2UobW9kZT0ibWFya2VycytsaW5lcyIsIG1hcmtlcj1saXN0KHNpemU9MTApLCBmcmFtZT1+c3ViX2lkLA0KICAgICAgICAgICAgbGluZT1saXN0KGNvbG9yPSJncmF5IikpICU+JSAjIGRhc2g9ImRvdCINCiAgaGlkZV9jb2xvcmJhcigpICU+JQ0KICBsYXlvdXQoc2hvd2xlZ2VuZD1GQUxTRSkNCg0KYWJfc3ViX2ZpZw0KYGBgDQoNCiMjIEEvQiBUZXN0IC0gU2F2ZSBTdWJqZWN0cyBGaWcNCg0KYGBge3J9DQpzYXZlV2lkZ2V0KGFiX3N1Yl9maWcsICJhYi1zdWJqZWN0cy1maWcuaHRtbCIpDQpgYGANCg0KIyBQZXJtdXRhdGlvbiBUZXN0DQoNCiMjIFBlcm11dGF0aW9uIFRlc3QgLSBEYXRhZnJhbWUgey5zbWFsbGVyfQ0KDQpgYGB7cn0NCnNldC5zZWVkKDApICMgRm9yIHJlcHJvZHVjaWJsZSByZXN1bHRzDQoNCnN1Yl9pZF8xID0gYyh3aGljaChhYl9yZXMgPT0gMSksIHdoaWNoKGFiX3JlcyA9PSAtMSkpDQpzdWJfY29sXzEgPSBjKHJlcCgxLCB0X3llcyksIHJlcCgtMSwgdF9ubykpDQoNCnBlcm1fcmFuZCA9IHNhbXBsZSgxOnQpDQpzdWJfaWRfMiA9IHN1Yl9pZF8xW3Blcm1fcmFuZF0NCnN1Yl9jb2xfMiA9IHN1Yl9jb2xfMVtwZXJtX3JhbmRdDQoNCiMgRGF0YSBmcmFtZSBmb3IgYW5pbWF0aW9ucw0KcGVybV9kZiA9IGRhdGEuZnJhbWUoDQogICMgUGVybSBzdGVwczogIFN0ZXAgMCwgICAgU3RlcCAxLCAgICBTdGVwIDINCiAgc3RlcCAgICA9IGMocmVwKDAsIHQpLCByZXAoMSwgdCksIHJlcCgyLCB0KSksICMgUGVybXV0YXRpb24gc3RlcA0KICBzdWJfeSAgID0gYyggICAgICAxOnQsICAgICAgIDE6dCwgICAgICAgMTp0KSwgIyBTdWJqZWN0IHkgcG9zaXRpb24NCiAgc3ViX2lkICA9IGMoICAgICAgMTp0LCAgc3ViX2lkXzEsICBzdWJfaWRfMiksICMgU3ViamVjdCBJRA0KICBzdWJfY29sID0gYyggICBhYl9yZXMsIHN1Yl9jb2xfMSwgc3ViX2NvbF8yKSAgIyBTdWJqZWN0IGNvbG9yIChBL0IgcmVzdWx0KQ0KKQ0KDQpwZXJtX2RmDQpgYGANCg0KIyMgUGVybXV0YXRpb24gVGVzdCAtIENyZWF0ZSBCYXNlIEZpZw0KDQpgYGB7cn0NCnNoYXBlcyA9IGxpc3QoDQogIGxpc3QodHlwZT0icmVjdCIsIHgwPS0wLjIsIHgxPTAuMiwgeTA9MC41LCAgeTE9MTAuNSksDQogIGxpc3QodHlwZT0icmVjdCIsIHgwPS0wLjIsIHgxPTAuMiwgeTA9MTAuNSwgIHkxPTIwLjUpLA0KICBsaXN0KHR5cGU9InJlY3QiLCB4MD0wLjgsIHgxPTEuMiwgeTA9MC41LCB5MT0yMC41KSwNCiAgbGlzdCh0eXBlPSJyZWN0IiwgeDA9MS44LCB4MT0yLjIsIHkwPTAuNSwgIHkxPTEwLjUpLA0KICBsaXN0KHR5cGU9InJlY3QiLCB4MD0xLjgsIHgxPTIuMiwgeTA9MTAuNSwgeTE9MjAuNSkNCikNCg0KYW5ub3RhdGlvbnMgPSBsaXN0KA0KICBsaXN0KHg9LTAuOCwgeT01LjUsIHRleHQ9IkE6IDcwJSIsIHNob3dhcnJvdz1GQUxTRSwgZm9udD1saXN0KHNpemU9MTgpKSwNCiAgbGlzdCh4PS0wLjgsIHk9MTUuNSwgdGV4dD0iQjogNDAlIiwgc2hvd2Fycm93PUZBTFNFLCBmb250PWxpc3Qoc2l6ZT0xOCkpLA0KICBsaXN0KHg9LTAuOCwgeT0xMC41LCB0ZXh0PSJBLUI6IDMwJSIsIHNob3dhcnJvdz1GQUxTRSwgZm9udD1saXN0KHNpemU9MTgpKSwNCiAgbGlzdCh4PTIuOCwgeT01LjUsIHRleHQ9IkE6IDUwJSIsIHNob3dhcnJvdz1GQUxTRSwgZm9udD1saXN0KHNpemU9MTgpKSwNCiAgbGlzdCh4PTIuOCwgeT0xNS41LCB0ZXh0PSJCOiA2MCUiLCBzaG93YXJyb3c9RkFMU0UsIGZvbnQ9bGlzdChzaXplPTE4KSksDQogIGxpc3QoeD0yLjgsIHk9MTAuNSwgdGV4dD0iQS1COiAtMTAlIiwgc2hvd2Fycm93PUZBTFNFLCBmb250PWxpc3Qoc2l6ZT0xOCkpDQopDQoNCnhfbGFiZWxzID0gbGlzdCgiU3RlcCAwXG5BL0IgUmVzdWx0cyIsICdTdGVwIDFcbiJCYWciJywgIlN0ZXBzIDItNVxuT25lIFBlcm11dGF0aW9uIikNCnhfYXhpcyA9IGxpc3QodGl0bGU9IiIsIHJhbmdlID0gYygtMS4zLCAzLjQpLCB0aWNrdmFscyA9IGxpc3QoMCwgMSwgMiksDQogICAgICAgICAgICAgIHRpY2t0ZXh0ID0geF9sYWJlbHMsIHplcm9saW5lPUZBTFNFLCBzaG93Z3JpZD1GQUxTRSkNCnlfYXhpcyA9IGxpc3QodGl0bGU9IiIsIHJhbmdlID0gYygwLCAyMSksIHplcm9saW5lPUZBTFNFLCBzaG93bGluZT1GQUxTRSwNCiAgICAgICAgICAgICAgc2hvd3RpY2tsYWJlbHM9RkFMU0UsIHNob3dncmlkID0gRkFMU0UpDQoNCnBlcm1fYmFzZV9maWcgPSBwZXJtX2RmICU+JQ0KICBwbG90X2x5KHg9fnN0ZXAsIHk9fnN1Yl95LCBjb2xvcj1+c3ViX2NvbCwgY29sb3JzPWMoInJlZCIsICJibGFjayIsICJncmVlbjMiKSkgJT4lDQogIGFkZF90cmFjZSh0eXBlPSJzY2F0dGVyIiwgbW9kZT0ibWFya2VycyIsIG1hcmtlcj1saXN0KHNpemU9MTApLCBvcGFjaXR5PTAuMykgJT4lDQogIGNvbmZpZyhkaXNwbGF5TW9kZUJhcj1GQUxTRSkgJT4lDQogIGxheW91dChhbm5vdGF0aW9ucz1hbm5vdGF0aW9ucywgbWFyZ2luPWxpc3QobD0wLCByPTAsIGI9MCwgdD0wLCBwYWQ9MCksIA0KICAgICAgICAgc2hhcGVzPXNoYXBlcywgeGF4aXM9eF9heGlzLCB5YXhpcz15X2F4aXMpDQoNCnBlcm1fYmFzZV9maWcNCmBgYA0KDQojIyBQZXJtdXRhdGlvbiBUZXN0IC0gQ3JlYXRlIFN0ZXBzIEZpZw0KDQpgYGB7cn0NCnBlcm1fc3RlcF9maWcgPSBwZXJtX2Jhc2VfZmlnICU+JQ0KICBhZGRfdHJhY2UodHlwZT0ic2NhdHRlciIsIG1vZGU9Im1hcmtlcnMiLCBtYXJrZXI9bGlzdChzaXplPTEwKSwgZnJhbWU9fnN0ZXAsIGlkcz1+c3ViX2lkKSAlPiUNCiAgaGlkZV9jb2xvcmJhcigpICU+JQ0KICBsYXlvdXQoc2hvd2xlZ2VuZD1GQUxTRSkNCg0KcGVybV9zdGVwX2ZpZw0KYGBgDQoNCiMjIFBlcm11dGF0aW8gVGVzdCAtIFNhdmUgU3RlcHMgRmlnDQoNCmBgYHtyfQ0Kc2F2ZVdpZGdldChwZXJtX3N0ZXBfZmlnLCAicGVybXV0YXRpb24tc3RlcHMtZmlnLmh0bWwiKQ0KYGBgDQoNCiMjIFBlcm11dGF0aW9uIFRlc3QgLSBDcmVhdGUgU3ViamVjdHMgRmlnDQoNCmBgYHtyfQ0KcGVybV9zdWJfZmlnID0gcGVybV9iYXNlX2ZpZyAlPiUNCiAgYWRkX3RyYWNlKHR5cGU9InNjYXR0ZXIiLCBtb2RlPSJtYXJrZXJzK2xpbmVzIiwgbWFya2VyPWxpc3Qoc2l6ZT0xMCksIGZyYW1lPX5zdWJfaWQpICU+JQ0KICBoaWRlX2NvbG9yYmFyKCkgJT4lDQogIGxheW91dChzaG93bGVnZW5kPUZBTFNFKQ0KDQpwZXJtX3N1Yl9maWcNCmBgYA0KDQojIyBQZXJtdXRhdGlvIFRlc3QgLSBTYXZlIFN1YmplY3RzIEZpZw0KDQpgYGB7cn0NCnNhdmVXaWRnZXQocGVybV9zdWJfZmlnLCAicGVybXV0YXRpb24tc3ViamVjdHMtZmlnLmh0bWwiKQ0KYGBgDQoNCg==